package crawler;

import com.alibaba.fastjson2.JSON;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.io.*;
import java.util.List;

import static java.lang.Thread.sleep;

public class get315Pic  implements PageProcessor {

    public static InputStream inStream = null;

    private Site site = Site.me()
            .setCharset("UTF-8")
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Safari/537.36")
            .setTimeOut(5000)
            .setRetryTimes(3)
            .setCycleRetryTimes(3)
            .setSleepTime(5000);

    @Override
    public void process(Page page) {
        Html html = page.getHtml();

        String s = html.xpath("//div[@class='show_cont block-info margin-top']//tbody//tr//td[1]//a//@href").regex("upload.*jpg").get();

        String s3="https://www.315jiage.cn/"+s;


        String baseurl = page.getUrl().get();









        //for (Selectable se:nodes) {
        //    String s1 = se.xpath("//td//text()").get();
        //    System.out.println(s1);
        //}


        //try {
        //    FileWriter fw = new FileWriter("a.txt",true);
        //    String line=html.toString();
        //    fw.write(line+"\r\n");
        //    fw.flush();//清空缓冲区
        //    fw.close();//关闭输出流
        //} catch (IOException e) {
        //    e.printStackTrace();
        //}

    }

    @Override
    public Site getSite() {
        return site;
    }


    public static void main(String[] args) {
        System.setProperty("selenuim_config", "conf/config.ini");
        Spider.create(new get315Pic())
                .addUrl("https://www.315jiage.cn/mn114182.aspx")	//要爬取的总链接
                .setDownloader(new SeleniumDownloader("C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe"))	//模拟启动浏览器
                //.setDownloader(new HttpClientDownloader())
                .thread(2)	//线程
                .run();
    }
}
